import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import matplotlib.pyplot as plt # for data visualization
import seaborn as sns # for statistical data visualization
%matplotlib inline
import warnings
warnings.filterwarnings('ignore')
import os

import plotly.express as px
import plotly.figure_factory as ff
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import MinMaxScaler
from sklearn.cluster import KMeans
from sklearn.manifold import TSNE
from mpl_toolkits.mplot3d import Axes3D
from sklearn.decomposition import PCA

live = pd.read_csv(r"C:\Users\acer\Downloads\Live.csv")

live = pd.read_csv(r"C:\Users\acer\Downloads\Live.csv")

live.head()

live.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 7050 entries, 0 to 7049
Data columns (total 12 columns):
 #   Column            Non-Null Count  Dtype 
---  ------            --------------  ----- 
 0   status_id         7050 non-null   object
 1   status_type       7050 non-null   object
 2   status_published  7050 non-null   object
 3   num_reactions     7050 non-null   int64 
 4   num_comments      7050 non-null   int64 
 5   num_shares        7050 non-null   int64 
 6   num_likes         7050 non-null   int64 
 7   num_loves         7050 non-null   int64 
 8   num_wows          7050 non-null   int64 
 9   num_hahas         7050 non-null   int64 
 10  num_sads          7050 non-null   int64 
 11  num_angrys        7050 non-null   int64 
dtypes: int64(9), object(3)
memory usage: 661.1+ KB

live.isnull().sum()

status_id           0
status_type         0
status_published    0
num_reactions       0
num_comments        0
num_shares          0
num_likes           0
num_loves           0
num_wows            0
num_hahas           0
num_sads            0
num_angrys          0
dtype: int64

len(live['status_id'].unique())

6997

len(live['status_published'].unique())

6913

live['status_type'].unique()

array(['video', 'photo', 'link', 'status'], dtype=object)

len(live['status_type'].unique())

4

live.drop(['status_id', 'status_published'], axis=1, inplace=True)

# Define the numerical columns
numerical_columns = ['num_reactions', 'num_comments', 'num_shares', 'num_likes', 'num_loves', 'num_wows', 'num_hahas', 'num_sads', 'num_angrys']

# Create a color palette
colors = sns.color_palette("husl", len(numerical_columns))

# Create subplots
fig, axes = plt.subplots(nrows=3, ncols=3, figsize=(21, 15))
axes = axes.flatten()

# Plot histograms
for i, column in enumerate(numerical_columns):
    sns.histplot(live[column], bins=50, ax=axes[i], color=colors[i], kde=True)
    axes[i].set_title(f'Distribution of {column}', fontsize=14)

# Adjust layout
plt.tight_layout()
plt.show()

# Define the numerical columns
numerical_columns = ['num_reactions', 'num_comments', 'num_shares', 'num_likes', 'num_loves', 'num_wows', 'num_hahas', 'num_sads', 'num_angrys']

# Compute the correlation matrix
corr_matrix = live[numerical_columns].corr()

# Compute the linkage matrix for hierarchical clustering
linkage_matrix = linkage(corr_matrix, method='ward')

# Create a clustermap
clustermap = sns.clustermap(
    corr_matrix,
    row_linkage=linkage_matrix,
    col_linkage=linkage_matrix,
    cmap='coolwarm',
    annot=True,  # Add annotations
    fmt='.2f',   # Format annotations to two decimal places
    linewidths=0.5,  # Add linewidths between cells
    figsize=(12, 10)  # Set the figure size
)

# Set title
plt.title('Hierarchical Clustering Heatmap', fontsize=16)
plt.show()

# Determine the number of pairs
num_pairs = len(numerical_columns) * (len(numerical_columns) - 1) // 2

# Create subplots
fig, axes = plt.subplots(nrows=(num_pairs // 3) + 1, ncols=3, figsize=(20, 17))
axes = axes.flatten()

# Plot scatter plots
pair_index = 0
for i in range(len(numerical_columns)):
    for j in range(i + 1, len(numerical_columns)):
        column1 = numerical_columns[i]
        column2 = numerical_columns[j]
        sns.scatterplot(x=live[column1], y=live[column2], ax=axes[pair_index])
        axes[pair_index].set_title(f'Relationship between {column1} and {column2}', fontsize=12)
        axes[pair_index].set_xlabel(column1, fontsize=10)
        axes[pair_index].set_ylabel(column2, fontsize=10)
        pair_index += 1

# Remove any empty subplots
for k in range(pair_index, len(axes)):
    fig.delaxes(axes[k])

# Adjust layout
plt.tight_layout()
plt.show()

categorical_columns = ['status_type']

# Set the style of the visualization
sns.set(style="whitegrid")

# Plot bar plots for categorical columns
for column in categorical_columns:
    plt.figure(figsize=(7, 5))
    ax = sns.countplot(data=live, x=column, palette="viridis")
    
    # Add count labels on top of the bars
    for p in ax.patches:
        ax.annotate(f'{p.get_height()}', 
                    (p.get_x() + p.get_width() / 2., p.get_height()), 
                    ha='center', va='center', 
                    xytext=(0, 10), 
                    textcoords='offset points',
                    fontsize=12)
    
    plt.title(f'Distribution of {column}', fontsize=16)
    plt.xlabel(column, fontsize=14)
    plt.ylabel('Count', fontsize=14)
    plt.xticks(rotation=45)
    plt.tight_layout()
    plt.show()

X = live
y = live['status_type']

le = LabelEncoder()
X['status_type'] = le.fit_transform(X['status_type'])
y = le.transform(y)

X.head()

cols = X.columns
ms = MinMaxScaler()
X = ms.fit_transform(X)
X = pd.DataFrame(X, columns=[cols])
X.head()

kmeans = KMeans(n_clusters=1, random_state=0) 

kmeans.fit(X)
labels = kmeans.labels_

# check how many of the samples were correctly labeled
correct_labels = sum(y == labels)

print("Result: %d out of %d samples were correctly labeled." % (correct_labels, y.size))
print('Accuracy score: {0:0.2f}'. format(correct_labels/float(y.size)))
print("Inertia", kmeans.inertia_)

Result: 63 out of 7050 samples were correctly labeled.
Accuracy score: 0.01
Inertia 70499.99999999974

kmeans = KMeans(n_clusters=3, random_state=0) 

kmeans.fit(X)
labels = kmeans.labels_

# check how many of the samples were correctly labeled
correct_labels = sum(y == labels)

print("Result: %d out of %d samples were correctly labeled." % (correct_labels, y.size))
print('Accuracy score: {0:0.2f}'. format(correct_labels/float(y.size)))
print("Inertia", kmeans.inertia_)

Result: 4066 out of 7050 samples were correctly labeled.
Accuracy score: 0.58
Inertia 44933.67071646415

kmeans = KMeans(n_clusters=4, random_state=0) 

kmeans.fit(X)
labels = kmeans.labels_

# check how many of the samples were correctly labeled
correct_labels = sum(y == labels)

print("Result: %d out of %d samples were correctly labeled." % (correct_labels, y.size))
print('Accuracy score: {0:0.2f}'. format(correct_labels/float(y.size)))
print("Inertia", kmeans.inertia_)

Result: 4061 out of 7050 samples were correctly labeled.
Accuracy score: 0.58
Inertia 41902.60200435425

import matplotlib.pyplot as plt
from sklearn.cluster import KMeans

# Define the range for the number of clusters
num_clusters_range = range(2, 11)

# Calculate WCSS for different numbers of clusters
wcss = []
for n_clusters in num_clusters_range:
    kmeans = KMeans(n_clusters=n_clusters, init='k-means++', random_state=42)
    kmeans.fit(X)
    wcss.append(kmeans.inertia_)

# Calculate the differences in WCSS to identify the elbow point
diff_scores = [wcss[i] - wcss[i + 1] for i in range(len(wcss) - 1)]

# Find the index of the maximum difference (elbow point)
elbow_index = diff_scores.index(max(diff_scores)) + 2  # Adjust index because we started from 2 clusters

# Print the optimal number of clusters
print(f"Optimal number of clusters (K-Means): {elbow_index}")

# Plot the Elbow Method
plt.figure(figsize=(12, 8))
plt.plot(num_clusters_range, wcss, marker='o', linestyle='-', color='#3498db', label='WCSS (Inertia)', linewidth=2)
plt.scatter(elbow_index, wcss[elbow_index - 2], s=300, c='#e74c3c', marker='*', label='Optimal Number of Clusters', zorder=10)
plt.title('The Elbow Method for Optimal Cluster Selection (K-Means)', fontsize=18)
plt.xlabel('Number of Clusters', fontsize=16)
plt.ylabel('Within-Cluster Sum of Squares (WCSS)', fontsize=16)
plt.grid(True, linestyle='--', alpha=0.7)
plt.legend(fontsize=14, loc='upper right')
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.show()

Optimal number of clusters (K-Means): 2

from sklearn.metrics import silhouette_score

# Define the range for the number of clusters
num_clusters_range = range(2, 11)

# Initialize lists to store silhouette scores
silhouette_scores = []

for n_clusters in num_clusters_range:
    kmeans = KMeans(n_clusters=n_clusters, init='k-means++', random_state=42)
    kmeans.fit(X)
    cluster_labels = kmeans.predict(X)
    silhouette_avg = silhouette_score(X, cluster_labels)
    silhouette_scores.append(silhouette_avg)

# Calculate the differences in silhouette scores
diff_scores = [silhouette_scores[i] - silhouette_scores[i + 1] for i in range(len(silhouette_scores) - 1)]

# Find the index of the maximum silhouette score (optimal clusters)
optimal_index = silhouette_scores.index(max(silhouette_scores)) + 2  # Adjust index because we started from 2 clusters

# Print the optimal number of clusters based on silhouette score
print(f"Optimal number of clusters (K-Means) based on Silhouette Score: {optimal_index}")

# Plot the Silhouette Scores
plt.figure(figsize=(12, 6))
plt.plot(num_clusters_range, silhouette_scores, marker='o', linestyle='-', color='#2ecc71', label='Silhouette Score', linewidth=2)
plt.scatter(optimal_index, silhouette_scores[optimal_index - 2], s=300, c='#e74c3c', marker='*', label='Optimal Number of Clusters', zorder=10)
plt.title('Silhouette Score for Optimal Cluster Selection (K-Means)', fontsize=18)
plt.xlabel('Number of Clusters', fontsize=16)
plt.ylabel('Silhouette Score', fontsize=16)
plt.grid(True, linestyle='--', alpha=0.7)
plt.legend(fontsize=14, loc='upper right')
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.show()

Optimal number of clusters (K-Means) based on Silhouette Score: 5

kmeans = KMeans(n_clusters=2, random_state=0) 
kmeans.fit(X)

KMeans(n_clusters=2, random_state=0)

KMeans(n_clusters=2, random_state=0)

labels = kmeans.labels_

# check how many of the samples were correctly labeled
correct_labels = sum(y == labels)

print("Result: %d out of %d samples were correctly labeled." % (correct_labels, y.size))
print('Accuracy score: {0:0.2f}'. format(correct_labels/float(y.size)))
print("Inertia", kmeans.inertia_)

Result: 4288 out of 7050 samples were correctly labeled.
Accuracy score: 0.61
Inertia 237.75726404419635

kmeans = KMeans(n_clusters=5, random_state=0) 
kmeans.fit(X)

KMeans(n_clusters=5, random_state=0)

KMeans(n_clusters=5, random_state=0)

labels = kmeans.labels_

# check how many of the samples were correctly labeled
correct_labels = sum(y == labels)

print("Result: %d out of %d samples were correctly labeled." % (correct_labels, y.size))
print('Accuracy score: {0:0.2f}'. format(correct_labels/float(y.size)))
print("Inertia", kmeans.inertia_)

Result: 4080 out of 7050 samples were correctly labeled.
Accuracy score: 0.58
Inertia 96.24989550305182

 #Sample data: Replace with your actual feature columns
feature_columns = ['num_reactions', 'num_comments']  # Adjust as needed
X = live[feature_columns]

# Normalize features
ms = MinMaxScaler()
X_normalized = ms.fit_transform(X)

# Initial centroids (Example values, replace with your specific initialization)
initial_centroid = np.array([[0.1, 0.1], [0.9, 0.9]])
plt.figure(figsize=(10, 6))
plt.scatter(X_normalized[:, 0], X_normalized[:, 1], c='gray', label='Data Points')
plt.scatter(initial_centroid[:, 0], initial_centroid[:, 1], s=200, c='black', label='Initial Centroids', marker='*')
plt.xlabel('Feature 1')
plt.ylabel('Feature 2')
plt.legend()
plt.title('Initial Centroid Initialization')
plt.show()

# K-Means Clustering with optimal number of clusters (2)
kmeans = KMeans(n_clusters=2, random_state=0, init=initial_centroid, n_init=1, max_iter=10)
kmeans.fit(X_normalized)
labels = kmeans.labels_

# Collecting centroid history
centroid_history = initial_centroid
for i in range(1, 11):
    kmeans = KMeans(n_clusters=2, random_state=0, init=initial_centroid, n_init=1, max_iter=i)
    kmeans.fit(X_normalized)
    centroid_history = np.vstack([centroid_history, kmeans.cluster_centers_])

    plt.figure(figsize=(9, 5))
    plt.scatter(X_normalized[labels == 0, 0], X_normalized[labels == 0, 1], c='yellow', label='Cluster 0')
    plt.scatter(X_normalized[labels == 1, 0], X_normalized[labels == 1, 1], c='purple', label='Cluster 1')

    plt.scatter(centroid_history[:, 0], centroid_history[:, 1], s=50, c='gray', label='Centroid Path', marker='x')
    plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1], s=200, c='black', label='Centroid', marker='*')

    plt.xlabel('Feature 1')
    plt.ylabel('Feature 2')
    plt.legend()
    plt.title(f'Iteration: {i} - Assignment and Move Centroid Step')
    plt.show()

# Plot centroid progress
def plot_k_means_progress(centroid_history, n_clusters, centroid_sets, cluster_color):
    for i in range(n_clusters):
        c_x = []
        c_y = []
        for j in range(centroid_sets):
            c_x.append(centroid_history[j * n_clusters + i, 0])
            c_y.append(centroid_history[j * n_clusters + i, 1])
        
        plt.plot(c_x, c_y, color=cluster_color['c_' + str(i)], linestyle='--', marker='o', label=f'Cluster {i}')

    plt.xlabel('Feature 1')
    plt.ylabel('Feature 2')
    plt.title('Centroid Path')
    plt.legend()
    plt.show()

# Define cluster colors
cluster_color = {'c_0': 'yellow', 'c_1': 'purple'}

# Plot centroid progress
plot_k_means_progress(centroid_history, 2, 11, cluster_color)

clusters = kmeans.fit_predict(X)
#Add cluster assignments to the DataFrame
live['Cluster'] = clusters

# Reduce dimensionality to 2D for visualization using PCA
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X)

# Create a DataFrame for the PCA components and cluster assignments
pca_df = pd.DataFrame(data=X_pca, columns=['PCA1', 'PCA2'])
pca_df['Cluster'] = clusters

# Plot the clusters
plt.figure(figsize=(9, 7))
sns.scatterplot(x='PCA1', y='PCA2', hue='Cluster', data=pca_df, palette='viridis', s=100, alpha=0.7, edgecolor='w')
plt.title('Clusters Visualization in 2D Space', fontsize=18)
plt.xlabel('PCA Component 1', fontsize=16)
plt.ylabel('PCA Component 2', fontsize=16)
plt.legend(title='Cluster', fontsize=14)
plt.grid(True, linestyle='--', alpha=0.7)
plt.show()

clusters = kmeans.fit_predict(X)

# Add cluster assignments to the DataFrame
live['Cluster'] = clusters

# Reduce dimensionality to 2D for visualization using t-SNE
tsne = TSNE(n_components=2, random_state=42)
X_tsne = tsne.fit_transform(X)

# Create a DataFrame for the t-SNE components and cluster assignments
tsne_df = pd.DataFrame(data=X_tsne, columns=['t-SNE1', 't-SNE2'])
tsne_df['Cluster'] = clusters

# Plot the clusters
plt.figure(figsize=(12, 8))
sns.scatterplot(x='t-SNE1', y='t-SNE2', hue='Cluster', data=tsne_df, palette='viridis', s=100, alpha=0.7, edgecolor='w')
plt.title('Clusters Visualization using t-SNE', fontsize=18)
plt.xlabel('t-SNE Component 1', fontsize=16)
plt.ylabel('t-SNE Component 2', fontsize=16)
plt.legend(title='Cluster', fontsize=14)
plt.grid(True, linestyle='--', alpha=0.7)
plt.show()

# PCA transformation
pca = PCA(n_components=3)
X_pca = pca.fit_transform(X)

# Create a DataFrame for PCA components and cluster assignments
pca_df = pd.DataFrame(data=X_pca, columns=['PCA1', 'PCA2', 'PCA3'])
pca_df['Cluster'] = clusters

# Get centroids and transform them to PCA space
centroids = kmeans.cluster_centers_
centroids_pca = pca.transform(centroids)

# Plotting
fig = plt.figure(figsize=(13, 13))
ax = fig.add_subplot(111, projection='3d')

# Scatter plot of the PCA transformed data
scatter = ax.scatter(pca_df['PCA1'], pca_df['PCA2'], pca_df['PCA3'], c=pca_df['Cluster'], cmap='viridis', s=100, alpha=0.7, edgecolor='w')

# Scatter plot of the centroids
ax.scatter(centroids_pca[:, 0], centroids_pca[:, 1], centroids_pca[:, 2], c='black', marker='*', s=300, label='Centroids', edgecolor='k')

ax.set_title('Clusters Visualization in 3D Space', fontsize=20)
ax.set_xlabel('PCA Component 1', fontsize=18)
ax.set_ylabel('PCA Component 2', fontsize=18)
ax.set_zlabel('PCA Component 3', fontsize=18)

# Add a legend
legend_elements = [
    plt.Line2D([0], [0], marker='o', color='w', label='Cluster 0', markerfacecolor='yellow', markersize=10),
    plt.Line2D([0], [0], marker='o', color='w', label='Cluster 1', markerfacecolor='purple', markersize=10),
    plt.Line2D([0], [0], marker='*', color='w', label='Centroid', markerfacecolor='black', markersize=15)
]
ax.legend(handles=legend_elements, title='Legend', fontsize=16, loc='upper right')

plt.grid(True, linestyle='--', alpha=0.7)
plt.show()

# t-SNE transformation
tsne = TSNE(n_components=3, random_state=42, perplexity=10, learning_rate=200)
X_tsne = tsne.fit_transform(X_normalized)

# Create a DataFrame for t-SNE components and cluster assignments
tsne_df = pd.DataFrame(data=X_tsne, columns=['t-SNE1', 't-SNE2', 't-SNE3'])
tsne_df['Cluster'] = clusters

# Get the centroids of the clusters and transform them into t-SNE space
centroids = kmeans.cluster_centers_
X_with_centroids = np.vstack([X_normalized, centroids])
tsne_with_centroids = TSNE(n_components=3, random_state=42, perplexity=10, learning_rate=200)
X_tsne_with_centroids = tsne_with_centroids.fit_transform(X_with_centroids)

# Extract the t-SNE transformation for centroids
centroids_tsne = X_tsne_with_centroids[-2:]

# Plotting
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(111, projection='3d')

# Scatter plot of the t-SNE transformed data
scatter = ax.scatter(tsne_df['t-SNE1'], tsne_df['t-SNE2'], tsne_df['t-SNE3'], c=tsne_df['Cluster'], cmap='viridis', s=150, alpha=0.8, edgecolor='w')

# Scatter plot of the centroids
ax.scatter(centroids_tsne[:, 0], centroids_tsne[:, 1], centroids_tsne[:, 2], c='black', marker='*', s=400, label='Centroids', edgecolor='k')

ax.set_title('Clusters Visualization in 3D Space using t-SNE', fontsize=24)
ax.set_xlabel('t-SNE Component 1', fontsize=20)
ax.set_ylabel('t-SNE Component 2', fontsize=20)
ax.set_zlabel('t-SNE Component 3', fontsize=20)

# Add a legend
legend_elements = [
    plt.Line2D([0], [0], marker='o', color='w', label='Cluster 0', markerfacecolor='yellow', markersize=15),
    plt.Line2D([0], [0], marker='o', color='w', label='Cluster 1', markerfacecolor='purple', markersize=15),
    plt.Line2D([0], [0], marker='*', color='w', label='Centroid', markerfacecolor='black', markersize=20)
]
ax.legend(handles=legend_elements, title='Legend', fontsize=18, loc='upper right')

plt.grid(True, linestyle='--', alpha=0.9)
plt.show()

from sklearn.neighbors import NearestNeighbors
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import DBSCAN

# Compute k-nearest neighbors distances
k = 5 
neigh = NearestNeighbors(n_neighbors=k)
neigh.fit(X)  # Fit the model

# Compute distances to the k-th nearest neighbor
distances, _ = neigh.kneighbors(X)
distances = distances[:, k-1]  # Extract distances to the k-th nearest neighbor

# Sort distances and plot
distances_sorted = np.sort(distances, axis=0)
plt.figure(figsize=(10, 6))
plt.plot(distances_sorted, marker='o', linestyle='-', color='#3498db')

# Highlight the "elbow" point in the plot
elbow_point = np.argmax(np.diff(distances_sorted, n=2)) + 1
plt.axvline(x=elbow_point, color='red', linestyle='--', label=f'Elbow Point ({elbow_point})')

plt.title(f'k-Distance Graph (k={k})', fontsize=18)
plt.xlabel('Points Sorted by Distance', fontsize=16)
plt.ylabel(f'{k}-th Nearest Neighbor Distance', fontsize=16)
plt.grid(True, linestyle='--', alpha=0.7)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.legend(fontsize=14)
plt.show()

# Print the suitable range for `eps`
print(f"Suitable range for `eps` based on the k-distance graph: {distances_sorted[elbow_point-1]} to {distances_sorted[-1]}")

Suitable range for `eps` based on the k-distance graph: 0.537642076119961 to 0.966388524547226

# Define a range of `eps` values based on the k-distance graph
eps_values = np.linspace(0.537642076119961, 0.966388524547226, 20)
num_clusters_dbscan_list = []

# Calculate the number of clusters for each `eps` value
for eps in eps_values:
    dbscan = DBSCAN(eps=eps, min_samples=5)
    clusters_dbscan = dbscan.fit_predict(X)
    num_clusters = len(set(clusters_dbscan)) - (1 if -1 in clusters_dbscan else 0)
    num_clusters_dbscan_list.append(num_clusters)

# Find the optimal `eps` based on the highest number of clusters
optimal_eps_index = num_clusters_dbscan_list.index(max(num_clusters_dbscan_list))
optimal_eps = eps_values[optimal_eps_index]

# Plot the Elbow Method for DBSCAN
plt.figure(figsize=(12, 8))
plt.plot(eps_values, num_clusters_dbscan_list, marker='o', linestyle='-', color='#3498db', label='Number of Clusters', linewidth=2)
plt.scatter(optimal_eps, num_clusters_dbscan_list[optimal_eps_index], s=300, c='#e74c3c', marker='*', zorder=10, label=f'Optimal `eps` ({optimal_eps:.2f})')
plt.title('Elbow Method for Optimal `eps` in DBSCAN', fontsize=18)
plt.xlabel('`eps` Value', fontsize=16)
plt.ylabel('Number of Clusters', fontsize=16)
plt.grid(True, linestyle='--', alpha=0.7)
plt.legend(fontsize=14, loc='upper right')
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.show()

# Print the optimal `eps` based on the highest number of clusters
print(f"Optimal `eps` for DBSCAN: {optimal_eps:.2f}")

# Print the optimal number of clusters for DBSCAN
optimal_num_clusters = max(num_clusters_dbscan_list)
print(f"Optimal number of clusters for DBSCAN: {optimal_num_clusters}")

Optimal `eps` for DBSCAN: 0.54
Optimal number of clusters for DBSCAN: 1

# Define and fit DBSCAN
dbscan = DBSCAN(eps=0.54, min_samples=5)
clusters_dbscan = dbscan.fit_predict(X)
X['Cluster_DBSCAN'] = clusters_dbscan

# Silhouette Score for K-Means
silhouette_kmeans = silhouette_score(X, kmeans_clusters)
print(f"Silhouette Score (K-Means): {silhouette_kmeans:.4f}")

# Silhouette Score for DBSCAN
if len(set(clusters_dbscan)) > 1:  # Check if there are any clusters other than noise
    silhouette_dbscan = silhouette_score(X, clusters_dbscan)
else:
    silhouette_dbscan = -1  # Not applicable if no clusters

print(f"Silhouette Score (DBSCAN): {silhouette_dbscan:.4f}")

Silhouette Score (K-Means): 0.7668
Silhouette Score (DBSCAN): 0.7741

# Visualize the Silhouette scores
sns.set_style("whitegrid")
plt.figure(figsize=(8, 6))
sns.barplot(x=['K-Means (k=2)', 'DBSCAN (k=1, eps =0.54)'], y=[silhouette_kmeans, silhouette_dbscan], palette= "viridis")
plt.xlabel('Algorithm')
plt.ylabel('Silhouette Score')
plt.title('Comparison of Silhouette Scores')
plt.ylim(0, 1)  # Set y-axis limits to [0, 1] for better visualization
plt.show()

 #Silhouette Score for K-Means
silhouette_kmeans = 0.7668

# Silhouette Score for DBSCAN
silhouette_dbscan = 0.7741

# Create a pie chart
plt.figure(figsize=(8, 6))
plt.pie([silhouette_kmeans, 1 - silhouette_kmeans, silhouette_dbscan, 1 - silhouette_dbscan],
        labels=['K-Means', ' ', 'DBSCAN', ' '],
        colors=['#F7DC6F', '#E5E5EA', '#8E44AD', '#E5E5EA'],
        autopct='%1.1f%%',
        startangle=90,
        pctdistance=0.85,
        labeldistance=1.1)
plt.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.
plt.title('Comparison of Silhouette Scores')
plt.show()

Step	Description
1. Explore the Dataset	Start by loading and analyzing the dataset to understand its features and structure. This step is crucial for preparing the data for clustering.
2. Initialize K-Means	Define the number of clusters K and randomly initialize the centroids. These centroids will serve as the initial points for the clustering process.
3. Assign Clusters & Optimize	Assign each data point to the nearest centroid by calculating distances. Determine the optimal number of clusters through analysis and evaluation.
4. Update Centroids	Recalculate the centroids by computing the mean position of all data points assigned to each cluster. This step updates the centroids' positions to better reflect the cluster distributions.
5. Visualize the Clusters	Create visualizations to examine the final clusters. Use distinct colors to represent different clusters and assess the effectiveness of the clustering algorithm.

	status_type	num_reactions	num_comments	num_shares	num_likes	num_loves	num_wows	num_hahas	num_sads
0	1.000000	0.112314	0.024393	0.076519	0.091720	0.140030	0.010791	0.006369	0.019608
1	0.333333	0.031847	0.000000	0.000000	0.031847	0.000000	0.000000	0.000000	0.000000
2	1.000000	0.048195	0.011243	0.016647	0.043312	0.031963	0.003597	0.006369	0.000000
3	0.333333	0.023567	0.000000	0.000000	0.023567	0.000000	0.000000	0.000000	0.000000
4	0.333333	0.045223	0.000000	0.000000	0.043312	0.013699	0.000000	0.000000	0.000000

Step	Description
1. 2D Clusters Visualization using PCA	Generate and plot 2D visualizations of clusters after applying Principal Component Analysis (PCA) to reduce dimensionality. Analyze how the clusters are distributed in the 2D PCA space.
2. 2D Clusters Visualization using t-SNE	Create 2D visualizations of clusters using t-SNE (t-Distributed Stochastic Neighbor Embedding) to preserve the local structure of the data and examine the clustering patterns in 2D space.
3. 3D Clusters Visualization using PCA	Plot clusters in a 3D space after applying PCA to capture the variance along three principal components. This helps in understanding the spatial distribution of clusters in a 3D context.
4. 3D Clusters Visualization using t-SNE	Visualize clusters in 3D after applying t-SNE to transform the data into three dimensions. This visualization helps in comprehending the cluster separations and structure in a 3D space.

Step	Description
1. Finding Suitable Range Using KNN	Use K-Nearest Neighbors (KNN) to determine the range of potential cluster numbers. This step involves evaluating various K values to find a suitable range for clustering.
2. Optimal EPS For DBSCAN by Elbow Method	Apply the Elbow Method to determine the optimal epsilon (EPS) parameter for DBSCAN. This method involves plotting the distance to the k-th nearest neighbor and finding the "elbow" point.
3. DBSCAN Model	Implement the DBSCAN clustering model with the chosen EPS value. This step involves fitting the model to the data and assigning cluster labels.
4. Models Comparison	Compare different clustering models by evaluating Silhouette Scores. Visualize the findings using bar plots and pie charts to assess and compare the clustering performance of various models.< sharply is selected./td>

Algorithm	Silhouette Score	Clustering Parameters
`K-Means`	0.7668	k=2
`DBSCAN`	0.7741	eps=0.54, min_samples=1

	status_id	status_type	status_published	num_reactions	num_comments	num_shares	num_likes	num_loves	num_wows	num_hahas	num_sads
0	246675545449582_1649696485147474	video	4/22/2018 6:00	529	512	262	432	92	3	1	1
1	246675545449582_1649426988507757	photo	4/21/2018 22:45	150	0	0	150	0	0	0	0
2	246675545449582_1648730588577397	video	4/21/2018 6:17	227	236	57	204	21	1	1	0
3	246675545449582_1648576705259452	photo	4/21/2018 2:29	111	0	0	111	0	0	0	0
4	246675545449582_1645700502213739	photo	4/18/2018 3:22	213	0	0	204	9	0	0	0

	status_type	num_reactions	num_comments	num_shares	num_likes	num_loves	num_wows	num_hahas	num_sads
0	3	529	512	262	432	92	3	1	1
1	1	150	0	0	150	0	0	0	0
2	3	227	236	57	204	21	1	1	0
3	1	111	0	0	111	0	0	0	0
4	1	213	0	0	204	9	0	0	0

	status_type	num_reactions	num_comments	num_shares	num_likes	num_loves	num_wows	num_hahas	num_sads
0	3	529	512	262	432	92	3	1	1
1	1	150	0	0	150	0	0	0	0
2	3	227	236	57	204	21	1	1	0
3	1	111	0	0	111	0	0	0	0
4	1	213	0	0	204	9	0	0	0

Cosmi Code ML Internship

Week- 3️⃣ Tasks By Abdullah Imran

Task No. I

K-Means¶

Section-1 : Exploring Dataset

Exploratory Data Analysis (EDA)

Preview the dataset¶

View summary of dataset¶¶

Check for missing values in dataset¶

Removing Un-Neccessary Columns¶

Visualizations Before Clustering

Histograms of Numerical Data

Hierarchical Clustering Heatmap¶

Pair-Plots¶

Distribution of Status-Type¶

Normalizing & Coverting Data

Declare feature vector and target variable¶

Feature Scaling¶

Section-2 : Initialize K-Means

K-Means model with One clusters¶

K-Means model with Three clusters¶

K-Means model with Four clusters¶

Section-3 : Finding Optimal Clusters

Elbow Method

The elbow method¶

Silhouette_Score

Using K-Means With Optimal Clusters

Elbow Method Optimal Number

Silhouette_Score Optimal Number

Results

Section-4 : Update Centroids

Visualizing Intial Centroids

Moving Centroids

Centroids Movement Path

Section-5 : Visualizing Clusters

1. 2D Clusters Visualization using PCA

2. 2D Clusters Visualization using T-SNE

3. 3D Clusters Visualization using PCA

3. 3D Clusters Visualization using T-SNE

Task No. III

Density-Based Spatial Clustering of Applications with Noise (DBSCAN)

Key Concepts and Hyperparameters¶

Key Terms¶

Hyperparameters¶

DBSCAN Clustering Algorithm¶

Hyperparameter Selection¶

Visualization¶

Summary¶

Section-1 : Finding Suitable Range Using KNN

Section-2 : Optimal EPS For DBCAN

Section-3 : DBCAN Model

Section-4 : MODELS COMPARISON

SILHOUETTE SCORES

BAR PLOT

PIE CHART REPRESENTATION

FINAL RESULTS

1. IF We Round off , Then The Both Models Have The Same Scores¶

2. IF Not, DBSCAN WINS¶

	status_type	num_reactions	num_comments	num_shares	num_likes	num_loves	num_wows	num_hahas	num_sads
0	3	529	512	262	432	92	3	1	1
1	1	150	0	0	150	0	0	0	0
2	3	227	236	57	204	21	1	1	0
3	1	111	0	0	111	0	0	0	0
4	1	213	0	0	204	9	0	0	0